# -*- coding: utf-8 -*-
import traceback
import sys, os
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.chrome.options import Options
import selenium.webdriver.support.ui as ui
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
from scrapy import Request
from lxml import etree
from selenium.webdriver.support.wait import WebDriverWait

from TycDetailParse.spiders.emailutils import send_email
from TycDetailParse.spiders.mysqlutils import MySqlTool
from TycDetailParse.spiders.parseinfo.company_info import get_company_info


class DetailparseSpider(scrapy.Spider):
    name = 'DetailParse'
    allowed_domains = ['www.tianyancha.com']
    start_urls = ['http://www.tianyancha.com/']

    def __init__(self):
        super(DetailparseSpider, self).__init__()  # 这里是关键
        chrome_options = Options()
        # chrome_options.add_argument('--headless')
        # chrome_options.add_argument('--disable-gpu')
        # chrome_options.add_argument('--no-sandbox')
        # chrome_options.add_argument('--disable-dev-shm-usage')
        self.driver = webdriver.Chrome(chrome_options=chrome_options)
        self.driver.get('https://www.tianyancha.com/')
        self.__tyc_province_instance = None  # 当前是爬取的哪一个省份的数据

    def parse(self, response):
        pass

    # ======================登录相关的代码===========================

    def start_requests(self):
        url = 'https://www.tianyancha.com/'
        yield Request(url, callback=self.login())

    def get_login(self):
        login_href = self.driver.find_element_by_xpath(
            '//*[@id="web-content"]/div/div[1]/div[1]/div/div/div[2]/div/div[4]/a')
        login_href.click()

    def switch_login(self):
        switch_href = self.driver.find_element_by_xpath(
            '//*[@class="modal-dialog -login-box animated"]/div/div[2]/div/div/div[3]/div[1]/div[2]')
        switch_href.click()

    def start_login(self):
        input1 = self.driver.find_element_by_xpath(
            '//*[@class="modal-dialog -login-box animated"]/div/div[2]/div/div/div[3]/div[2]/div[2]/input')  # 手机号码
        input2 = self.driver.find_element_by_xpath(
            '//*[@class="modal-dialog -login-box animated"]/div/div[2]/div/div/div[3]/div[2]/div[3]/input')  # 密码
        input1.send_keys('18649019096')
        input2.send_keys('jhj198606')
        # 17040838437, eeee5545
        # 13022649621, mima1234
        # 15243812410, mima1234
        time.sleep(1)
        login_button = self.driver.find_element_by_xpath(
            '//*[@class="modal-dialog -login-box animated"]/div/div[2]/div/div/div[3]/div[2]/div[5]')  # 点击登录
        login_button.click()

    # 登录相关的操作
    def login(self):
        time.sleep(1)
        self.get_login()
        time.sleep(1)
        self.switch_login()
        time.sleep(1)
        self.start_login()
        time.sleep(8)
        self.after_login()

    # 登录成功之后，读取数据库中的url，挨个去获取公司详情信息，并解析数据
    def after_login(self):
        time.sleep(1)

        # 正常情况，走true的逻辑，读取数据库url，并挨个解析公司详情信息，并保存
        if True:
            self.get_detail_url()
            # 程序退出发送邮件
            self.send_email()
        else:
            # 直接run这个函数，测试解析通过url获取公司详情并解析
            self.get_detail_info_for_debug()

    # 读取数据库里面的url
    def get_detail_url(self):
        try:
            page_number = 1  # 从第一页开始
            number_per_page = 1000  # 每页取1000个

            is_last_page = False  # 是否是最后一页

            mysqlutil = MySqlTool()
            # 获取会话指针
            with mysqlutil.conn.cursor() as cursor:

                while True:

                    if is_last_page:
                        break

                    # 查询sql语句
                    # numberperpage 表示每页要显示的条数，pagenumber表示页码，那么 返回第pagenumber页，每页条数为numberperpage的sql语句：
                    # select * from company_url limit (pagenumber-1)*numberperpage,numberperpage
                    # select * from company_url limit numberperpage offset (pagenumber-1)*numberperpage
                    offset = (page_number - 1) * number_per_page
                    sql_limit = 'select company_url from tyc_company_list_tianjin limit %s offset %s' %(number_per_page, offset) ## 此处tyc_company_list_tianjin需要修改为对应的表

                    count = cursor.execute(sql_limit)
                    print("=====================" + 'count is ' + count + "=====================")

                    if count < number_per_page:
                        is_last_page = True
                        page_number = page_number + 1
                    else:
                        page_number = page_number + 1

                    print("=====================" + 'page_number is ' + page_number + "=====================")

                    # 提交
                    mysqlutil.conn.commit()
                    # 查询数据
                    # 方案1
                    # result = cursor.fetchall()

                    # 方案2
                    # result = cursor.fetchmany(3)
                    # print(result)
                    # print(result[0]["company_url"])

                    # 方案3
                    while True:
                        result = cursor.fetchone()
                        if not result:
                            break
                        print(result["company_url"])
                        # TODO 此处要解决崩溃问题
                        get_company_info(self.driver, result["company_url"])
                        # 延迟一秒之后再去读取
                        time.sleep(1)
        except Exception as e:
            print(e)
        finally:
            mysqlutil.conn.close()

    # 这个函数专门用来debug数据使用的
    def get_detail_info_for_debug(self):
        url = "https://www.tianyancha.com/company/59837300"  # 手动填写url，自己去天眼查官方网站查找数据并修改此处
        get_company_info(self.driver, url)

    # 程序异常退出的时候，发送邮件
    def send_email(self):
        # 打印异常信息，并发送邮件
        msg = traceback.format_exc()
        print(msg)
        send_email("程序运行完毕，退出了！！！！！！")


if __name__ == '__main__':
    # 直接run这个函数，测试解析通过url获取公司详情并解析
    detailparseSpider = DetailparseSpider()
    detailparseSpider.get_detail_info_for_debug()
